import java.io.IOException;
import java.util.ArrayList;

public class Program {

	public static final int X = 7;
	public static final int Y = 7;
	public static final float ALFA = 0.1f;
	public static final float GAMMA = 0.9f;
	public static final float EPSILON = 0.1f;
	public static final int STANDARD_REWARD = -1;
	public static final int GOAL_REWARD = 10;
	public static final int COLLISION_REWARD = -10;
	private static final int nbOfEpisodes = 5000;
	private static GridWorld world;

	public static void main(String[] args) {
		// for (int i = 0; i < 100; i++) {
		world = new GridWorld(X, Y);
		ArrayList<Agent> agents = new ArrayList<Agent>();
		agents.add(new Agent(ALFA, GAMMA, EPSILON, world, new Coordinate(1, 0),
				new Coordinate(5, 6), STANDARD_REWARD, GOAL_REWARD,
				COLLISION_REWARD, "agent1.txt"));
		agents.add(new Agent(ALFA, GAMMA, EPSILON, world, new Coordinate(5, 0),
				new Coordinate(1, 6), STANDARD_REWARD, GOAL_REWARD,
				COLLISION_REWARD, "agent2.txt"));

		doQLearning(agents);

		learnCoordination(agents);

		testCQPolicy(agents);

		// for (Agent agent : agents) {
		// System.out.println(agent.teller);
		// }

		// }
		System.out.println("Done!");
	}

	private static void testCQPolicy(ArrayList<Agent> agents) {
		for (Agent agent : agents) {
			agent.enterStartState();
		}

		boolean done = false;
		while (!done) {
			for (Agent agent : agents) {
				if (!agent.isInGoal()) {
					agent.takeNormalStep();
				}
			}
			done = true;
			for (Agent agent : agents) {
				if (!agent.isInGoal()) {
					done = false;
				}
			}
		}
		for (Agent agent : agents) {
			try {
				agent.out.flush();
				agent.out.close();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}

	private static void learnCoordination(ArrayList<Agent> agents) {
		// Na elke ronde robots deftig resetten! zowel positie in robot van goal
		// naar start als
		// het feit of een state een robot bevat
		for (int i = 0; i < nbOfEpisodes; i++) {
			for (Agent agent : agents) {
				agent.enterStartState();
			}
			boolean done = false;
			while (!done) {
				for (Agent agent : agents) {
					if (!agent.isInGoal()) {
						agent.takeCoordinatedLearningStep();
					}
				}
				done = true;
				for (Agent agent : agents) {
					if (!agent.isInGoal()) {
						done = false;
					}
				}
			}
			for (Agent agent : agents) {
				agent.reset();
			}
		}
	}

	private static void doQLearning(ArrayList<Agent> agents) {
		for (Agent agent : agents) {
			for (int i = 0; i < nbOfEpisodes; i++) {
				while (!agent.isInGoal())
					agent.takeSingleLearningStep();
				agent.reset();
			}
			// System.out.println(agent.bestDirections());
			// ArrayList<String> pathToGoal = agent.getPathToGoal();
			// for (String path : pathToGoal)
			// System.out.print(path + " ");
			// Utility.writePath(pathToGoal);
			// System.out.println();
		}
	}
}
